package com.xiaotu.spider.project.pageprocessor;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 IQYDocPageProcessor
 * @日期 2017年4月5日
 * @作者 高海军
 * @功能 爱奇艺纪录片爬取
 */
@Component("IQYDocPageProcessor")
public class IQYDocPageProcessor extends AUrlPageProcessor
{
	protected static final String CSV_SPLITOR = "#***#";
	
	protected static final String LABEL = "?label=";
	
	@Override
	public void process(Page page)
	{
		if (page.getUrl().toString().contains("list.iqiyi.com"))
		{
			this.listProcess(page);
			this.addNextPage(page);
		}
		else
			detailProcess(page);
	}
	
	protected void listProcess(Page page)
	{
		String url = page.getUrl().toString();
		page.getRequest().getExtras().put("label",
				url.substring(url.length() - "汉字".length()));
		List<String> list = page.getHtml()
				.xpath("//div[@class='wrapper-piclist']//div[@class='site-piclist_pic']")
				.all();
		list.stream().filter(divStr -> this.docFilter(divStr))
				.forEach(divStr -> PageUtils.addTargetRequest(page,
						new Html(divStr).xpath("//a/@href").get(), page
								.getRequest().getExtras()));
	}
	
	protected boolean docFilter(String divStr)
	{
		return !RegexUtils.regexMatch("更新至([0-9]{4}-[0-9]{2}-[0-9]{2})期",
				new Html(divStr).xpath("//span[@class='icon-vInfo']/text()")
						.get().trim());
	}
	
	protected void detailProcess(Page page)
	{
		String[] dataArr = this.getBaseInfo(page);
		if (dataArr == null)
			return;
		String line = StringUtils.join(dataArr, CSV_SPLITOR);
		System.out.println(line);
		FileUtil.writeFile("D://iqy_探索.csv", line + SepratorUtil.LINE_SEPRATOR);
	}
	
	protected String[] getBaseInfo(Page page)
	{
		Html html = page.getHtml();
		String title = html.xpath("//h1[@class='main_title']/a/text()").get();
		if (StringUtils.isEmpty(title))
			title = html.xpath("//h2[@class='h2Title1']/strong/a/text()").get();
		if (StringUtils.isEmpty(title))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		String year = html.xpath("//h1[@class='main_title']/span/text()").get();
		String play = html.xpath("//a[@class='allplayNums-type-a']/i/text()")
				.get();
		if (StringUtils.isEmpty(play))
			play = html.xpath("//span[@id='playCount']/text()").get();
		String desc = html
				.xpath("//span[@data-moreorless='moreinfo']/span/text()").get();
		if (StringUtils.isEmpty(desc))
			desc = html.xpath("//span[@data-moreorless='lessinfo']/span/text()")
					.get();
		if (StringUtils.isEmpty(desc))
			desc = html.xpath("//p[@class='twP']").get();
		desc = StringUtils.isEmpty(desc) ? StringUtils.EMPTY
				: HtmlUtil.delHTMLTag(desc);
		String score = this.getScore(html);
		
		Map<String, String> fieldMap = getFields(html);
		String[] fieldKeys = this.getFieldKey();
		return new String[] { page.getRequest().getExtra("label").toString(),
				title, year,
				StringUtils.isEmpty(fieldMap.get(fieldKeys[0]))
						? StringUtils.EMPTY : fieldMap.get(fieldKeys[0]),
				StringUtils.isEmpty(fieldMap.get(fieldKeys[1]))
						? html.xpath("//span[@class='twW50']/a/text()").get()
						: fieldMap.get(fieldKeys[1]),
				StringUtils.isEmpty(fieldMap.get(fieldKeys[2]))
						? StringUtils.EMPTY
						: this.getIntVal(fieldMap.get(fieldKeys[2]), page),
				play, score, desc };
	}
	
	protected String getIntVal(String val, Page page)
	{
		return RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, val);
	}
	
	protected String[] getFieldKey()
	{
		return new String[] { "地区", "类型", "集数" };
	}
	
	protected Map<String, String> getFields(Html html)
	{
		List<String> list = html
				.xpath("//div[@class='topic_item clearfix']/div").all();
		Map<String, String> map = new HashMap<>();
		list.forEach(divStr ->
		{
			Html divObj = new Html(divStr);
			String key = divObj.xpath("//em/text()").regex(RegexUtils.REGEX_CH)
					.get();
			String value = StringUtils
					.join(divObj.xpath("//em/a/text()").all().toArray(),
							SepratorUtil.SEP_COMMA_EN)
					.trim();
			map.put(key, value);
		});
		return map;
	}
	
	@SuppressWarnings("unchecked")
	protected String getScore(Html html)
	{
		String dataId = this.getDataId(html);
		if (StringUtils.isEmpty(dataId))
			return StringUtils.EMPTY;
		String data = HttpUtils.httpGetJSONP(
				"http://up-video.iqiyi.com/ugc-updown/quud.do?dataid=" + dataId
						+ "&type=1&userid=&flashuid=e8b46504e032b0423e14e98822596337&appID=21&callback=window.Q.__callbacks__.cbsfr1oe");
		if (StringUtils.isEmpty(data))
			return StringUtils.EMPTY;
		data = data.substring(data.indexOf("(") + "(".length(),
				data.indexOf(")"));
		Map<String, Object> map;
		try
		{
			map = GsonUtils.fromJson(data, Map.class);
		}
		catch (Exception e)
		{
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		}
		Map<String, Object> dataMap = (Map<String, Object>) map.get("data");
		if (dataMap == null)
			return StringUtils.EMPTY;
		return dataMap.get("score") == null ? StringUtils.EMPTY
				: dataMap.get("score").toString();
	}
	
	protected String getDataId(Html html)
	{
		String dataId = html
				.xpath("//a[@class='allplayNums-type-a']/i/@data-playcount-albumid")
				.get();
		if (StringUtils.isEmpty(dataId))
		{
			dataId = RegexUtils.regexSearch(
					"\"albumId\":(" + RegexUtils.REGEX_INTEGER + ")",
					html.get());
			dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, dataId);
		}
		return dataId;
	}
	
	protected void addNextPage(Page page)
	{
		String href = page.getHtml()
				.xpath("//div[@class='mod-page']/a[@data-key='down']/@href")
				.get();
		String label = page.getUrl().toString()
				.substring(page.getUrl().toString().indexOf(LABEL));
		if (StringUtils.isNotEmpty(href))
		{
			if (!href.contains("label="))
				href += label;
			PageUtils.addTargetRequest(page, href,
					page.getRequest().getExtras());
		}
		
	}
	
	@Override
	protected String[] getStartRequestUrls()
	{
		return new String[] {
				"http://list.iqiyi.com/www/3/73-------------4-1--iqiyi-1-.html?label=探索"// ,
				// "http://list.iqiyi.com/www/3/77-------------4-1--iqiyi-1-.html?label=文化",
				// "http://list.iqiyi.com/www/3/310-------------4-1--iqiyi-1-.html?label=旅游"
		};
	}
	
	@Override
	protected String getSpiderType()
	{
		return "IQYDoc";
	}
	
	@Override
	public String getPageDataType(Page page)
	{
		return Constants.PageDataType.SEARCH_LIST;
	}
	
}
